import scrapy
from bs4 import BeautifulSoup

from Konachan.items import KonachanItem


class KonachanSpider(scrapy.Spider):
    name = 'mio'
    allowed_domains = ['konachan.com']
    start_urls = ['https://konachan.com/post']

    def parse(self, response):
        html = response.body
        bs = BeautifulSoup(html, 'lxml')
        img_link = []

        for link in bs.find_all(class_='directlink largeimg'):
            img_link.append(link.get('href'))

        item = KonachanItem()
        item['image_urls'] = img_link
        yield item

        next_url = 'https://konachan.com' + bs.find(class_='next_page').get('href')
        if next_url:
            yield scrapy.Request(next_url, callback=self.parse)
